{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 420,
     "status": "ok",
     "timestamp": 1638340822379,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "5fnibDWxzzrt",
    "outputId": "0885d642-478c-4b44-e65d-a8f6fa5c0383",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20698,
     "status": "ok",
     "timestamp": 1638340843074,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "T-4s0PVgzz7t",
    "outputId": "4690bb9a-0c96-4794-d900-3268cab8d28a",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "%reload_ext watermark\n",
    "%watermark -v -p numpy,pandas,torch,transformers"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20894,
     "status": "ok",
     "timestamp": 1638340863965,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "dPdpNulr5rdc",
    "outputId": "12875265-a502-405b-c62e-2e60324b740d",
    "scrolled": true,
    "is_executing": true
   },
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "#@title Setup & Config\n",
    "import transformers\n",
    "from transformers import BertModel, BertTokenizer, AdamW, get_linear_schedule_with_warmup\n",
    "import torch\n",
    "import unicodedata\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from pylab import rcParams\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import rc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "from collections import defaultdict\n",
    "from textwrap import wrap\n",
    "\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn.functional as F\n",
    "from sklearn.metrics import f1_score\n",
    "from torch.nn import TransformerEncoder, TransformerEncoderLayer\n",
    "from collections import Counter\n",
    "import ast\n",
    "import random\n",
    "import json\n",
    "from math import isnan\n",
    "\n",
    "\n",
    "%matplotlib inline\n",
    "%config InlineBackend.figure_format='retina'\n",
    "\n",
    "sns.set(style='whitegrid', palette='muted', font_scale=1.2)\n",
    "\n",
    "HAPPY_COLORS_PALETTE = [\"#01BEFE\", \"#FFDD00\", \"#FF7D00\", \"#FF006D\", \"#ADFF02\", \"#8F00FF\"]\n",
    "\n",
    "sns.set_palette(sns.color_palette(HAPPY_COLORS_PALETTE))\n",
    "\n",
    "rcParams['figure.figsize'] = 12, 8\n",
    "\n",
    "RANDOM_SEED = 10\n",
    "np.random.seed(RANDOM_SEED)\n",
    "torch.manual_seed(RANDOM_SEED)\n",
    "\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 2200,
     "status": "ok",
     "timestamp": 1638340866156,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "pEHKZt-qRG5x",
    "outputId": "142ea35d-c3c7-4ffb-9ff9-15d174ef5312",
    "is_executing": true
   },
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HHAlx_DgsuY6"
   },
   "source": [
    "## New Section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 177,
     "referenced_widgets": [
      "cce639937ece4bd082588d5b592a4a0d",
      "0aa1acd28f504292b91d7c32ee7d64ab",
      "f660e8b44e204795afd2d0449886e37a",
      "a2ac63819f2144c5b66b57132f1ff185",
      "9bdfbf15bf7345f5847736122acacd7b",
      "37984a156b634c668ac59e1ee50ab3c4",
      "233a52b14ada464a993ec8aee9d31f29",
      "b42983f2583e40c1bf3c398cef8e2cf5",
      "ae07d83052cf46ae9c4231eebcb8c5d4",
      "676a15c6b28e47f69d65e837314ac142",
      "2c95ebb1f47a4113b9eba6753e248081",
      "1c66a9e829f74978a6dc6c5831c26d66",
      "040c33afcdff47fcb1b807543c48eeb1",
      "ad27528a6d144adfbe3dbc276ee43298",
      "1205434242214938b9b58ccf2f940300",
      "af1d28bcf3814a978da6cb77bfe2537c",
      "3aa458d8906e48daace7aed0292bd6f3",
      "c1b1e62f31564c3d801a835ef6f723d1",
      "7f27bc2760854988ae083dcdd738c33c",
      "a74e5e1c50254c5ab459718dc2c59fb7",
      "c485b6952a4a421f89383b43b7002fd8",
      "590af74a6be4442e8daa38c85df146e0",
      "6abc245ff8454df4a8f82ba73b8b9dc8",
      "ba73aa97f348474185b341310767fb21",
      "b9422f74176244e69952d394491fec8e",
      "fe53e4c52b8f4ab198d221f2432b8798",
      "102f2b2c193340c4bbb10595aae790c5",
      "4f903879853d4bdb89a83608e71b79b5",
      "b8b6f228c2dc42519de4137d19f24e3a",
      "aa469eccce834677bff79b2d7080ea0f",
      "59bbb69619154d6f807d924cbd057972",
      "da2eb0187a44429696ec150e27a7c559",
      "3fd4f746e7cf4bdcb5b85322ff8eb5b8",
      "c4db4ab5d5b142c787da7e737f425050",
      "182deebee91d429db286768ffd8f330b",
      "6f4e06b72b5b445a8e7402e8b269b063",
      "46695f3c7b3b443194eac3d83918c812",
      "1636be77b6e447af9fe0795d63620d44",
      "a0b7ea994efa410bb22796fedf6af172",
      "914e53be08704e638acbdf37f5d06dcc",
      "ab071e2b77a84e41a17b95671a3b1ad8",
      "66a007fb557848aab6f41102a1b8305f",
      "9a433ec7dd0a4626b1f28e41804f0e5f",
      "36a3d6d220f8419ba1482110bac0cbe4",
      "b25c54179da848acaceec7f396da6b09",
      "2758357c443e4c6aa54b11cd7aed859b",
      "44767259a6da4a82bdbdae29787c52c9",
      "7c07929736284b658849f60af1b674d3",
      "3671d85e40434594b04539775f8f52dd",
      "83082bfcc08e43578a245146fd226427",
      "eaffde3cf6ff4ee0a8e1aeade4a2db8e",
      "69e640e5d0eb447c8f4eb78cfe538e32",
      "25d2a9565b5f422f980da2f0dde46232",
      "5575420737784e09916ad4194a761dac",
      "0fdde7e1bc0f49c7a2a54364588408fb"
     ]
    },
    "executionInfo": {
     "elapsed": 4407,
     "status": "ok",
     "timestamp": 1638340870560,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "xRdWcGGBFYTu",
    "outputId": "b2f148f6-6973-4efe-db93-35b902202690",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer,BertModel\n",
    "PRE_TRAINED_MODEL_NAME = \"hfl/chinese-roberta-wwm-ext\"\n",
    "# PRE_TRAINED_MODEL_NAME = \"hfl/chinese-roberta-wwm-ext-large\"\n",
    "# PRE_TRAINED_MODEL_NAME = \"hfl/chinese-bert-wwm-ext\"\n",
    "# PRE_TRAINED_MODEL_NAME = \"bert-base-uncased\"\n",
    "# PRE_TRAINED_MODEL_NAME = \"ProsusAI/finbert\"\n",
    "# PRE_TRAINED_MODEL_NAME = \"bert-base-chinese\"\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME,do_lower_case=True)\n",
    "PRE_TRAINED_MODEL_NAME = '../ROBERT_4_model.bin'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "eg7yz0zXQujX"
   },
   "source": [
    "## Generating the masks of verb, A0, A1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 41060,
     "status": "ok",
     "timestamp": 1638340911616,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "c4joMCxJKR5a",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('../data/train.csv',sep='\\t')\n",
    "df_val = pd.read_csv('../data/val.csv',sep='\\t')\n",
    "df_test = pd.read_csv('../data/test.csv',sep='\\t')\n",
    "df_ood = pd.read_csv('../data/ood.csv',sep='\\t')\n",
    "\n",
    "df_train = df_train.drop(df_train.loc[df_train.verbA0A1.isna()].index)\n",
    "df_test = df_test.drop(df_test.loc[df_test.verbA0A1.isna()].index)\n",
    "df_val = df_val.drop(df_val.loc[df_val.verbA0A1.isna()].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1.isna()].index)\n",
    "\n",
    "df_train = df_train.drop(df_train.loc[df_train.verbA0A1=='[]'].index)\n",
    "df_test = df_test.drop(df_test.loc[df_test.verbA0A1=='[]'].index)\n",
    "df_val = df_val.drop(df_val.loc[df_val.verbA0A1=='[]'].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1=='[]'].index)\n",
    "\n",
    "def string_to_tuples_list(text):\n",
    "  if text is np.nan or text =='[]':\n",
    "    return []\n",
    "  text = ''.join(text.split('], ['))\n",
    "  tmp = eval(text.strip('[').strip(']'))\n",
    "  if not isinstance(tmp[0],tuple):\n",
    "    return [tmp]\n",
    "  return list(tmp)\n",
    "\n",
    "for col in ['verb','A0','A1']:\n",
    "  df_train[col] = df_train[col].apply(string_to_tuples_list)\n",
    "  df_val[col] = df_val[col].apply(string_to_tuples_list)\n",
    "  df_test[col] = df_test[col].apply(string_to_tuples_list)\n",
    "  df_ood[col] = df_ood[col].apply(string_to_tuples_list)\n",
    "\n",
    "for col in ['stock_factors','verbA0A1']:\n",
    "# for col in ['verbA0A1']:\n",
    "  df_train[col] = df_train[col].apply(ast.literal_eval)\n",
    "  df_val[col] = df_val[col].apply(ast.literal_eval)\n",
    "  df_test[col] = df_test[col].apply(ast.literal_eval)\n",
    "  df_ood[col] = df_ood[col].apply(ast.literal_eval)\n",
    "\n",
    "def mask(df):\n",
    "  df = df.reset_index(drop = True)\n",
    "  df['verb_mask'] = 0\n",
    "  df['A0_mask'] = 0\n",
    "  df['A1_mask'] = 0\n",
    "  df['verb_mask'] = df['verb_mask'].astype('object')\n",
    "  df['A0_mask'] = df['A0_mask'].astype('object')\n",
    "  df['A1_mask'] = df['A1_mask'].astype('object')\n",
    "  for index,row in df.iterrows():\n",
    "\n",
    "    df.at[index,'stock_factors'] = [*map(float,df.loc[index,'stock_factors'])]\n",
    "    AV_num = 0\n",
    "    for k,col in enumerate(['verb','A0','A1']):\n",
    "      masks = []\n",
    "      for j in range(len(row['verbA0A1'])):\n",
    "        mask = np.zeros(299)\n",
    "        idx = []\n",
    "        for v in row['verbA0A1'][j][k]:\n",
    "          \n",
    "          idx = idx + [int(i) for i in range(v[0],v[0]+v[1])]\n",
    "        # idx = np.unique(idx).tolist()\n",
    "        counter = Counter(idx)\n",
    "\n",
    "        mask = [0 if counter[i]== 0 else 1/len(counter) for i in range(0,len(mask))]\n",
    "        mask.insert(0,0)\n",
    "        masks.append(mask)\n",
    "      AV_num = len(masks)\n",
    "      for i in range(10 - len(masks)):\n",
    "        masks.append(np.zeros(300))\n",
    "      while len(masks)>10:\n",
    "        masks.pop()\n",
    "      name = col+'_mask'\n",
    "      df.at[index,name] = np.array(masks)\n",
    "    if AV_num>10:\n",
    "      AV_num=10\n",
    "    df.loc[index,'AV_num'] = int(AV_num)\n",
    "  df.AV_num = df.AV_num.astype('int')\n",
    "  df.stock_factors = df.stock_factors.apply(np.array)\n",
    "  return df\n",
    "\n",
    "\n",
    "df_train = mask(df_train)\n",
    "df_test = mask(df_test)\n",
    "df_val = mask(df_val)\n",
    "df_ood = mask(df_ood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false,
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# max(df_train.verb_mask.apply(len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# df_train.to_csv('./data/df_train_srl.csv',sep='\\t',index=None)\n",
    "# df_test.to_csv('./data/df_test_srl.csv',sep='\\t',index=None)\n",
    "# df_val.to_csv('./data/df_val_srl.csv',sep='\\t',index=None)\n",
    "# df_ood.to_csv('./data/df_ood_srl.csv',sep='\\t',index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "sns.countplot(df_train.label)\n",
    "plt.xlabel('review score')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1638340911616,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "ixHO5Iujxbx-",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "total = pd.concat([df_train,df_test,df_val])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 759,
     "status": "ok",
     "timestamp": 1638340912366,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "1WfYpnhaxbwG",
    "outputId": "055f0756-aefa-4600-d5b0-28baa5d40d80",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "total.sort_values(by='DATE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1638340912366,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "BASpBDwaxbtx",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1638340912366,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "0NlS1_q1xbqx",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ajOfqpcWRBt9"
   },
   "source": [
    "## Creat Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1638340912367,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "WaKd7JaTy2gw",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "max_len = 300\n",
    "class_names = ['negative','neutral', 'positive']\n",
    "class GPReviewDataset(Dataset):\n",
    "\n",
    "  def __init__(self, reviews, targets,verb,A0,A1,AV_num,tokenizer,stock_factors, max_len):\n",
    "    self.reviews = reviews\n",
    "    self.targets = targets\n",
    "    self.stock_factors = stock_factors\n",
    "    self.verb = verb\n",
    "    self.A0 = A0\n",
    "    self.A1 = A1\n",
    "    self.AV_num = AV_num\n",
    "    self.tokenizer = tokenizer\n",
    "    self.max_len = max_len\n",
    "  \n",
    "  def __len__(self):\n",
    "    return len(self.reviews)\n",
    "  \n",
    "  def __getitem__(self, item):\n",
    "    review = str(self.reviews[item])\n",
    "    target = self.targets[item]\n",
    "    stock_factors = self.stock_factors[item]\n",
    "    v = self.verb[item]\n",
    "    a0 = self.A0[item]\n",
    "    a1 = self.A1[item]\n",
    "    av_num = self.AV_num[item]\n",
    "\n",
    "    encoding = self.tokenizer.encode_plus(\n",
    "      review,\n",
    "      add_special_tokens=True,\n",
    "      max_length=self.max_len,\n",
    "      return_token_type_ids=False,\n",
    "      padding='max_length',\n",
    "      truncation=True,\n",
    "      return_attention_mask=True,\n",
    "      return_tensors='pt',\n",
    "    )\n",
    "\n",
    "    return {\n",
    "      'review_text': review,\n",
    "      'input_ids': encoding['input_ids'].flatten(),\n",
    "      'attention_mask': encoding['attention_mask'].flatten(),\n",
    "      'targets': torch.tensor(target, dtype=torch.long),\n",
    "      'stock_factors':torch.tensor(stock_factors),\n",
    "      'verb': torch.tensor(v),\n",
    "      'A0': torch.tensor(a0),\n",
    "      'A1': torch.tensor(a1),\n",
    "      'AV_num': torch.tensor(av_num)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1638340912367,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "cPZSIigpPtEr",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "def create_data_loader(df, tokenizer, max_len, batch_size):\n",
    "  ds = GPReviewDataset(\n",
    "    reviews=df.text_a.to_numpy(),\n",
    "    targets=df.label.to_numpy(),\n",
    "    stock_factors = df.stock_factors,\n",
    "    verb = df.verb_mask,\n",
    "    A0 = df.A0_mask,\n",
    "    A1 = df.A1_mask,\n",
    "    AV_num = df.AV_num,\n",
    "    tokenizer=tokenizer,\n",
    "    max_len=max_len\n",
    "  )\n",
    "\n",
    "  return DataLoader(\n",
    "    ds,\n",
    "    batch_size=batch_size,\n",
    "    # num_workers=4,\n",
    "    shuffle=True\n",
    "  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 10,
     "status": "ok",
     "timestamp": 1638340912367,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "5D6CwQDIAk0C",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "\n",
    "df_train = df_train.reset_index(drop=True)\n",
    "df_test = df_test.reset_index(drop=True)\n",
    "df_val = df_val.reset_index(drop=True)\n",
    "df_ood = df_ood.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 11,
     "status": "ok",
     "timestamp": 1638340912368,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "upOSL-NiRhzy",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 2  # TODO： batch_size大小需根据显卡改\n",
    "\n",
    "train_data_loader = create_data_loader(df_train, tokenizer, max_len, BATCH_SIZE)\n",
    "val_data_loader = create_data_loader(df_val, tokenizer, max_len, BATCH_SIZE)\n",
    "test_data_loader = create_data_loader(df_test, tokenizer, max_len, BATCH_SIZE)\n",
    "ood_data_loader = create_data_loader(df_ood, tokenizer, max_len, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "kxjZ9xW9exg8"
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 538,
     "status": "ok",
     "timestamp": 1638340912896,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "m6tsxogmBNWm",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "NUMBER_FACTOR = 24\n",
    "class SentimentClassifier(nn.Module):\n",
    "\n",
    "  def __init__(self, n_classes):\n",
    "    super(SentimentClassifier, self).__init__()\n",
    "    self.bert = BertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)\n",
    "    self.encoder_layer =  nn.TransformerEncoderLayer(d_model=2304+NUMBER_FACTOR, nhead=1)\n",
    "    self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, num_layers=2)\n",
    "    self.drop = nn.Dropout(p=0.1)\n",
    "    # self.L1 = nn.Linear(self.bert.config.hidden_size*30, self.bert.config.hidden_size*3)\n",
    "    self.out1 = nn.Linear((self.bert.config.hidden_size*3+NUMBER_FACTOR)*10, (self.bert.config.hidden_size*3+NUMBER_FACTOR)*3)\n",
    "    self.out = nn.Linear((self.bert.config.hidden_size*3+NUMBER_FACTOR)*3, n_classes)\n",
    "    self.linear_for_stock_factors = nn.Linear(NUMBER_FACTOR,NUMBER_FACTOR)\n",
    "    self.flatten2 = nn.Flatten(2,-1)\n",
    "    self.flatten = nn.Flatten(1,-1)\n",
    "    self.relu = nn.ReLU()\n",
    "    self.sig = nn.Sigmoid()\n",
    "    self.Querry = nn.Linear(self.bert.config.hidden_size*2,self.bert.config.hidden_size*2)\n",
    "    self.Key = nn.Linear(self.bert.config.hidden_size*2,self.bert.config.hidden_size*2)\n",
    "    # self.Self_supervised = nn.Linear(self.bert.config.hidden_size,self.bert.config.hidden_size)\n",
    "\n",
    "  \n",
    "  def forward(self, input_ids, attention_mask,verb,A0,A1,stock_factors,AV_num):\n",
    "    #get bert embedding \n",
    "    hidden_state = self.bert(\n",
    "      input_ids=input_ids,\n",
    "      attention_mask=attention_mask\n",
    "    )[0]\n",
    "    batch = hidden_state.shape[0]\n",
    "    verb_tmp = verb.clone()\n",
    "\n",
    "    #stock_factor(8*24)\n",
    "    stock_factors_batch = stock_factors[:,:NUMBER_FACTOR]\n",
    "    stock_factors_batch = self.linear_for_stock_factors(stock_factors_batch.float())\n",
    "    stock_factors_batch = self.relu(stock_factors_batch)\n",
    "    stock_factors_batch = torch.unsqueeze(stock_factors_batch,1) #8*1*2304\n",
    "    stock_factors_batch= torch.cat(10*[stock_factors_batch],1)\n",
    "\n",
    "    A0_tmp = A0.clone()\n",
    "    A1_tmp = A1.clone()\n",
    "    #mask verb\n",
    "    AV_idx = []\n",
    "    for idx,num in enumerate(AV_num):\n",
    "      self_label = torch.randint(0,num,(1,))[0]\n",
    "      AV_idx.append(self_label)\n",
    "      A0_tmp[idx,self_label,:]=0\n",
    "      A1_tmp[idx,self_label,:]=0\n",
    "\n",
    "   #verb label\n",
    "    AV_idx = torch.tensor(AV_idx).to(device)\n",
    "\n",
    "    # get K(8*10*1536)\n",
    "    A0_mask = torch.unsqueeze(A0,3)\n",
    "    A0_mask = torch.cat(768 * [A0_mask],3)\n",
    "    A0_mask = torch.mean(A0_mask*torch.unsqueeze(hidden_state,1),2,True)\n",
    "    \n",
    "    A1_mask = torch.unsqueeze(A1,3)\n",
    "    A1_mask = torch.cat(768 * [A1_mask],3)\n",
    "    A1_mask = torch.mean(A1_mask*torch.unsqueeze(hidden_state,1),2,True)\n",
    "    K = self.Key(torch.squeeze(torch.cat([A0_mask,A1_mask],3)).float())\n",
    "    \n",
    "    # get verb embedding after masking(8*10*1*768)\n",
    "    V_mask = torch.unsqueeze(verb,3)\n",
    "    V_mask = torch.cat(768 * [V_mask],3)\n",
    "    V_mask = torch.mean(V_mask*torch.unsqueeze(hidden_state,1),2,True)\n",
    "    transformer_input = V_mask\n",
    "\n",
    "    #get A0 embedding(8*10*2*768)\n",
    "    A0_mask_mask = torch.unsqueeze(A0_tmp,3)\n",
    "    A0_mask_mask = torch.cat(768 * [A0_mask_mask],3)\n",
    "    A0_mask_mask = torch.mean(A0_mask_mask*torch.unsqueeze(hidden_state,1),2,True)\n",
    "    transformer_input = torch.cat([transformer_input,A0_mask_mask],2)\n",
    "\n",
    "    #get A1 embedding(8*10*3*768)\n",
    "    A1_mask_mask = torch.unsqueeze(A1_tmp,3)\n",
    "    A1_mask_mask = torch.cat(768 * [A1_mask_mask],3)\n",
    "    A1_mask_mask = torch.mean(A1_mask_mask*torch.unsqueeze(hidden_state,1),2,True)\n",
    "    transformer_input = torch.cat([transformer_input,A1_mask_mask],2)\n",
    "\n",
    "    #get transformer input(8*10*2304)\n",
    "    transformer_input = self.flatten2(transformer_input.float())\n",
    "    transformer_input = torch.cat([transformer_input,stock_factors_batch],2)\n",
    "\n",
    "\n",
    "    #turn to (11*8*2304)\n",
    "    transformer_input = torch.stack([transformer_input[:,i,:] for i in range(0,len(verb[0]))])\n",
    "    \n",
    "    #get transformer output(11*8*2304)\n",
    "    transformer_output = self.transformer_encoder(transformer_input)\n",
    "\n",
    "    #turn to (8*11*2304)\n",
    "    transformer_output = torch.stack([torch.squeeze(transformer_output[:,i,:]) for i in range(0,batch)])\n",
    "    transformer_output = torch.squeeze(transformer_output)\n",
    "\n",
    "    #turn to (8*11*2304)\n",
    "    self_pred = torch.zeros((batch,10))\n",
    "    if transformer_output.dim()==2:\n",
    "      transformer_output = torch.unsqueeze(transformer_output,0)\n",
    "\n",
    "    #Q For CL\n",
    "    for idx,i in enumerate(AV_idx):\n",
    "      Q = torch.unsqueeze(self.Querry(transformer_output[idx,i,768:2304]),0)\n",
    "      self_pred[idx]=(Q @ K[idx].T)\n",
    "\n",
    "    #get transformer input for classification(8*10*3*768)\n",
    "    transformer_input = torch.cat([V_mask,A0_mask,A1_mask],2)\n",
    "\n",
    "    #get transformer input(8*11*2304)\n",
    "    transformer_input = self.flatten2(transformer_input.float())\n",
    "    transformer_input = torch.cat([transformer_input,stock_factors_batch],2)\n",
    "\n",
    "\n",
    "    #get transformer output(11*8*2304)\n",
    "    transformer_input = torch.stack([transformer_input[:,i,:] for i in range(0,len(transformer_input[0]))])\n",
    "    transformer_output = self.transformer_encoder(transformer_input)\n",
    "\n",
    "    #turn to (8*11*2304)\n",
    "    transformer_output = torch.stack([torch.squeeze(transformer_output[:,i,:]) for i in range(0,batch)])\n",
    "    transformer_output = torch.squeeze(transformer_output)\n",
    "\n",
    "    # handle exception\n",
    "    if transformer_output.dim()==2:\n",
    "      transformer_output = torch.unsqueeze(transformer_output,0)\n",
    "\n",
    "    output = self.flatten(transformer_output.float())\n",
    "    output = self.sig(output)\n",
    "#     output = self.drop(output)\n",
    "    output = self.out1(output)\n",
    "    output = self.sig(output)\n",
    "    output = self.drop(output)\n",
    "    output = self.out(output)\n",
    "    \n",
    "    return output,self_pred,AV_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 20147,
     "status": "ok",
     "timestamp": 1638340933036,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "KQEOV2pfYBPs",
    "outputId": "8c72de76-8c42-4431-bed8-99f90be6ded5",
    "scrolled": true,
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "model = SentimentClassifier(3)\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1638340933037,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "4q2fUr9TYMxW",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "EPOCHS = 10\n",
    "\n",
    "optimizer = AdamW(model.parameters(), lr=1e-5, correct_bias=False)\n",
    "total_steps = len(train_data_loader) * EPOCHS\n",
    "\n",
    "scheduler = get_linear_schedule_with_warmup(\n",
    "  optimizer,\n",
    "  num_warmup_steps=2,\n",
    "  num_training_steps=total_steps\n",
    ")\n",
    "\n",
    "loss_fn = nn.CrossEntropyLoss().to(device)\n",
    "# loss_fn = nn.KLDivLoss(reduce=False,size_average=False).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1638340933037,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "oLLFWDvDYeeS",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "def train_epoch(\n",
    "  model, \n",
    "  data_loader, \n",
    "  loss_fn, \n",
    "  optimizer, \n",
    "  device, \n",
    "  scheduler, \n",
    "  n_examples\n",
    "):\n",
    "  model = model.train()\n",
    "\n",
    "  losses = []\n",
    "  correct_predictions = 0\n",
    "  correct_predictions_verbs = 0\n",
    "\n",
    "  for d in data_loader:\n",
    "    input_ids = d[\"input_ids\"].to(device)\n",
    "    attention_mask = d[\"attention_mask\"].to(device)\n",
    "    targets = d[\"targets\"].to(device)\n",
    "    stock_factors = d['stock_factors'].to(device)\n",
    "    verb = d[\"verb\"].to(device)\n",
    "    A0 = d[\"A0\"].to(device)\n",
    "    A1 = d[\"A1\"].to(device)\n",
    "    AV_num = d[\"AV_num\"].to(device)\n",
    "\n",
    "    outputs,self_outputs,self_labels = model(\n",
    "      input_ids=input_ids,\n",
    "      attention_mask=attention_mask,\n",
    "      verb = verb,\n",
    "      A0 = A0,\n",
    "      A1 = A1,\n",
    "      stock_factors = stock_factors,\n",
    "      AV_num = AV_num\n",
    "    )\n",
    "    self_outputs = self_outputs.to(device)\n",
    "    self_labels = self_labels.to(device)\n",
    "\n",
    "\n",
    "\n",
    "    if outputs.dim()==1:\n",
    "      outputs = torch.unsqueeze(outputs,0)\n",
    "    _, preds = torch.max(outputs, dim=1)\n",
    "    _, self_preds = torch.max(self_outputs, dim=1)\n",
    "\n",
    "    # print(targets.shape,outputs.shape)\n",
    "    loss = 0.8*loss_fn(outputs, targets)+0.2*loss_fn(self_outputs,self_labels)\n",
    "\n",
    "    correct_predictions += torch.sum(preds == targets)\n",
    "    correct_predictions_verbs += torch.sum(self_preds == self_labels)\n",
    "    losses.append(loss.item())\n",
    "\n",
    "    loss.backward()\n",
    "    nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
    "    optimizer.step()\n",
    "    scheduler.step()\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "  return correct_predictions.double() / n_examples, np.mean(losses),correct_predictions_verbs.double()/n_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1638340933038,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "VmDYxiXDYl-u",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "def eval_model(model, data_loader, loss_fn, device, n_examples):\n",
    "  model = model.eval()\n",
    "\n",
    "  losses = []\n",
    "  correct_predictions = 0\n",
    "\n",
    "  with torch.no_grad():\n",
    "    for d in data_loader:\n",
    "      input_ids = d[\"input_ids\"].to(device)\n",
    "      attention_mask = d[\"attention_mask\"].to(device)\n",
    "      targets = d[\"targets\"].to(device)\n",
    "      stock_factors = d['stock_factors'].to(device)\n",
    "      verb = d[\"verb\"].to(device)\n",
    "      A0 = d[\"A0\"].to(device)\n",
    "      A1 = d[\"A1\"].to(device)\n",
    "      AV_num = d[\"AV_num\"].to(device)\n",
    "\n",
    "      outputs = model(\n",
    "        input_ids=input_ids,\n",
    "        attention_mask=attention_mask,\n",
    "        verb = verb,\n",
    "        A0 = A0,\n",
    "        A1 = A1,\n",
    "        stock_factors = stock_factors,\n",
    "        AV_num = AV_num\n",
    "      )[0]\n",
    "      \n",
    "      if outputs.dim()==1:\n",
    "        outputs = torch.unsqueeze(outputs,0)\n",
    "      _, preds = torch.max(outputs, dim=1)\n",
    "      \n",
    "      loss = loss_fn(outputs, targets)\n",
    "\n",
    "      correct_predictions += torch.sum(preds == targets)\n",
    "      \n",
    "      losses.append(loss.item())\n",
    "\n",
    "  return correct_predictions.double() / n_examples, np.mean(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1638340933038,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "Nkvo-2p93Dnr",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "history = defaultdict(list)\n",
    "best_accuracy = 0\n",
    "best_train_accuracy = 0\n",
    "# EPOCHS = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 547
    },
    "executionInfo": {
     "elapsed": 2422,
     "status": "error",
     "timestamp": 1638340935442,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "VZvKK2TkYojZ",
    "outputId": "f12e9df1-fa0d-4a0d-f3f9-9b5b7411adc4",
    "scrolled": true,
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "\n",
    "  print(f'Epoch {epoch + 1}/{EPOCHS}')\n",
    "  print('-' * 10)\n",
    "\n",
    "  train_acc, train_loss, cl_acc = train_epoch(\n",
    "    model,\n",
    "    train_data_loader,    \n",
    "    loss_fn, \n",
    "    optimizer, \n",
    "    device, \n",
    "    scheduler, \n",
    "    len(df_train)\n",
    "  )\n",
    "\n",
    "  print(f'Train loss {train_loss} accuracy {train_acc} cl_accuracy{cl_acc}')\n",
    "\n",
    "  if train_acc > best_train_accuracy:\n",
    "    torch.save(model.state_dict(), 'best_train_model_state.bin')\n",
    "    best_train_accuracy = train_acc\n",
    "    print(\"save to best train model\")\n",
    "\n",
    "  val_acc, val_loss = eval_model(\n",
    "    model,\n",
    "    val_data_loader,\n",
    "    loss_fn, \n",
    "    device, \n",
    "    len(df_val)\n",
    "  )\n",
    "\n",
    "  print(f'Val   loss {val_loss} accuracy {val_acc}')\n",
    "  print()\n",
    "\n",
    "  history['train_acc'].append(train_acc)\n",
    "  history['train_loss'].append(train_loss)\n",
    "  history['val_acc'].append(val_acc)\n",
    "  history['val_loss'].append(val_loss)\n",
    "\n",
    "  if val_acc > best_accuracy:\n",
    "    torch.save(model.state_dict(), 'Factor24_Pretrained_RoBert_SRL(FC)_A0A1.bin')\n",
    "    best_accuracy = val_acc\n",
    "    print(\"save to best model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 19,
     "status": "aborted",
     "timestamp": 1638340935429,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "xu7Y2BPzLZgi",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "plt.plot([i.cpu() for i in history['train_acc']], label='train accuracy')\n",
    "plt.plot([i.cpu() for i in history['val_acc']], label='validation accuracy')\n",
    "\n",
    "plt.title('Training history')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.xlabel('Epoch')\n",
    "plt.legend()\n",
    "plt.ylim([0, 1]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 20,
     "status": "aborted",
     "timestamp": 1638340935430,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "YYMhWiIrBrV4",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "model = SentimentClassifier(len(class_names))\n",
    "model.load_state_dict(torch.load('Factor24_Pretrained_RoBert_SRL(FC)_A0A1.bin'))\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 20,
     "status": "aborted",
     "timestamp": 1638340935430,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "c7Cckbz8GbDK",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "test_acc, _ = eval_model(\n",
    "  model,\n",
    "  test_data_loader,\n",
    "  loss_fn,\n",
    "  device,\n",
    "  len(df_test)\n",
    ")\n",
    "\n",
    "test_acc.item()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 21,
     "status": "aborted",
     "timestamp": 1638340935431,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "X17uMr0xbOvW",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 21,
     "status": "aborted",
     "timestamp": 1638340935431,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "a0Qemnl0FiEb",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "def get_predictions(model, data_loader):\n",
    "  model = model.eval()\n",
    "  \n",
    "  review_texts = []\n",
    "  predictions = []\n",
    "  prediction_probs = []\n",
    "  real_values = []\n",
    "\n",
    "  with torch.no_grad():\n",
    "    for d in data_loader:\n",
    "\n",
    "      texts = d[\"review_text\"]\n",
    "      input_ids = d[\"input_ids\"].to(device)\n",
    "      attention_mask = d[\"attention_mask\"].to(device)\n",
    "      targets = d[\"targets\"].to(device)\n",
    "      verb = d[\"verb\"].to(device)\n",
    "      stock_factors = d['stock_factors'].to(device)\n",
    "      A0 = d[\"A0\"].to(device)\n",
    "      A1 = d[\"A1\"].to(device)\n",
    "      AV_num = d['AV_num'].to(device)\n",
    "\n",
    "      outputs = model(\n",
    "        input_ids=input_ids,\n",
    "        attention_mask=attention_mask,\n",
    "        verb = verb,\n",
    "        A0 = A0,\n",
    "        A1 = A1,\n",
    "        stock_factors = stock_factors,\n",
    "        AV_num = AV_num\n",
    "      )[0]\n",
    "      _, preds = torch.max(outputs, dim=1)\n",
    "\n",
    "      probs = F.softmax(outputs, dim=1)\n",
    "\n",
    "      review_texts.extend(texts)\n",
    "      predictions.extend(preds)\n",
    "      prediction_probs.extend(probs)\n",
    "      real_values.extend(targets)\n",
    "\n",
    "  predictions = torch.stack(predictions).cpu()\n",
    "  prediction_probs = torch.stack(prediction_probs).cpu()\n",
    "  real_values = torch.stack(real_values).cpu()\n",
    "  return review_texts, predictions, prediction_probs, real_values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "y_review_texts, y_pred, y_pred_probs, y_test = get_predictions(\n",
    "  model,\n",
    "  test_data_loader\n",
    ")\n",
    "\n",
    "# df = pd.DataFrame([y_review_texts, y_pred.numpy(), y_pred_probs.numpy(), y_test.numpy()]).T\n",
    "# df = df.rename(columns={0:'text',1:'prediction',2:'probability',3:'labels'})\n",
    "# df.to_csv('/content/drive/MyDrive/data/df_prediction_roberta_srl_cl_f.csv',sep='\\t')\n",
    "\n",
    "print(classification_report(y_test, y_pred, target_names=class_names,digits=4))\n",
    "\n",
    "def show_confusion_matrix(confusion_matrix):\n",
    "  hmap = sns.heatmap(confusion_matrix, annot=True, fmt=\"d\", cmap=\"Blues\")\n",
    "  hmap.yaxis.set_ticklabels(hmap.yaxis.get_ticklabels(), rotation=0, ha='right')\n",
    "  hmap.xaxis.set_ticklabels(hmap.xaxis.get_ticklabels(), rotation=30, ha='right')\n",
    "  plt.ylabel('True sentiment')\n",
    "  plt.xlabel('Predicted sentiment');\n",
    "\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "df_cm = pd.DataFrame(cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_cm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOD Result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "y_ood_review_texts, y_ood_pred, y_ood_pred_probs, y_ood = get_predictions(\n",
    "  model,\n",
    "  ood_data_loader\n",
    ")\n",
    "\n",
    "print(classification_report(y_ood, y_ood_pred, target_names=class_names,digits=4))\n",
    "\n",
    "ood_cm = confusion_matrix(y_ood, y_ood_pred)\n",
    "df_ood_cm = pd.DataFrame(ood_cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_ood_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "df_ood = pd.read_csv('../data/ood.csv',sep='\\t')\n",
    "df_ood = df_ood.loc[(df_ood.DATE>='2021-01-01')&(df_ood.DATE<='2021-03-31')]\n",
    "df_ood = df_ood.sort_values(by='DATE')\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "# df_ood = df_ood.loc[:4000]\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1.isna()].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1=='[]'].index)\n",
    "for col in ['verb','A0','A1']:\n",
    "  df_ood[col] = df_ood[col].apply(string_to_tuples_list)\n",
    "\n",
    "for col in ['stock_factors','verbA0A1']:\n",
    "  df_ood[col] = df_ood[col].apply(ast.literal_eval)\n",
    "df_ood = mask(df_ood)\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "ood_data_loader = create_data_loader(df_ood, tokenizer, max_len, BATCH_SIZE)\n",
    "\n",
    "y_ood_review_texts, y_ood_pred, y_ood_pred_probs, y_ood = get_predictions(\n",
    "  model,\n",
    "  ood_data_loader\n",
    ")\n",
    "\n",
    "print(classification_report(y_ood, y_ood_pred, target_names=class_names,digits=4))\n",
    "\n",
    "ood_cm = confusion_matrix(y_ood, y_ood_pred)\n",
    "df_ood_cm = pd.DataFrame(ood_cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_ood_cm)\n",
    "plt.show()\n",
    "\n",
    "df_ood = pd.read_csv('../data/pre/ood.csv',sep='\\t')\n",
    "df_ood = df_ood.loc[(df_ood.DATE>='2021-04-01')&(df_ood.DATE<='2021-06-31')]\n",
    "df_ood = df_ood.sort_values(by='DATE')\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "# df_ood = df_ood.loc[:4000]\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1.isna()].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1=='[]'].index)\n",
    "for col in ['verb','A0','A1']:\n",
    "  df_ood[col] = df_ood[col].apply(string_to_tuples_list)\n",
    "\n",
    "for col in ['stock_factors','verbA0A1']:\n",
    "  df_ood[col] = df_ood[col].apply(ast.literal_eval)\n",
    "df_ood = mask(df_ood)\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "ood_data_loader = create_data_loader(df_ood, tokenizer, max_len, BATCH_SIZE)\n",
    "\n",
    "y_ood_review_texts, y_ood_pred, y_ood_pred_probs, y_ood = get_predictions(\n",
    "  model,\n",
    "  ood_data_loader\n",
    ")\n",
    "\n",
    "print(classification_report(y_ood, y_ood_pred, target_names=class_names,digits=4))\n",
    "\n",
    "ood_cm = confusion_matrix(y_ood, y_ood_pred)\n",
    "df_ood_cm = pd.DataFrame(ood_cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_ood_cm)\n",
    "plt.show()\n",
    "\n",
    "df_ood = pd.read_csv('../data/pre/ood.csv',sep='\\t')\n",
    "df_ood = df_ood.loc[(df_ood.DATE>='2021-07-01')&(df_ood.DATE<='2021-09-31')]\n",
    "df_ood = df_ood.sort_values(by='DATE')\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "# df_ood = df_ood.loc[:4000]\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1.isna()].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1=='[]'].index)\n",
    "for col in ['verb','A0','A1']:\n",
    "  df_ood[col] = df_ood[col].apply(string_to_tuples_list)\n",
    "\n",
    "for col in ['stock_factors','verbA0A1']:\n",
    "  df_ood[col] = df_ood[col].apply(ast.literal_eval)\n",
    "df_ood = mask(df_ood)\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "ood_data_loader = create_data_loader(df_ood, tokenizer, max_len, BATCH_SIZE)\n",
    "\n",
    "y_ood_review_texts, y_ood_pred, y_ood_pred_probs, y_ood = get_predictions(\n",
    "  model,\n",
    "  ood_data_loader\n",
    ")\n",
    "\n",
    "print(classification_report(y_ood, y_ood_pred, target_names=class_names,digits=4))\n",
    "\n",
    "ood_cm = confusion_matrix(y_ood, y_ood_pred)\n",
    "df_ood_cm = pd.DataFrame(ood_cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_ood_cm)\n",
    "plt.show()\n",
    "\n",
    "df_ood = pd.read_csv('../data/pre/ood.csv',sep='\\t')\n",
    "df_ood = df_ood.loc[df_ood.DATE>='2021-10-01']\n",
    "df_ood = df_ood.sort_values(by='DATE')\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "# df_ood = df_ood.loc[:4000]\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1.isna()].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1=='[]'].index)\n",
    "for col in ['verb','A0','A1']:\n",
    "  df_ood[col] = df_ood[col].apply(string_to_tuples_list)\n",
    "\n",
    "for col in ['stock_factors','verbA0A1']:\n",
    "  df_ood[col] = df_ood[col].apply(ast.literal_eval)\n",
    "df_ood = mask(df_ood)\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "ood_data_loader = create_data_loader(df_ood, tokenizer, max_len, BATCH_SIZE)\n",
    "\n",
    "y_ood_review_texts, y_ood_pred, y_ood_pred_probs, y_ood = get_predictions(\n",
    "  model,\n",
    "  ood_data_loader\n",
    ")\n",
    "\n",
    "print(classification_report(y_ood, y_ood_pred, target_names=class_names,digits=4))\n",
    "\n",
    "ood_cm = confusion_matrix(y_ood, y_ood_pred)\n",
    "df_ood_cm = pd.DataFrame(ood_cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_ood_cm)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "df_ood = pd.read_csv('../data/df_all_year_srl.csv',sep='\\t')\n",
    "# df_ood = df_ood.loc[(df_ood.DATE>='2021-05-05')&(df_ood.DATE<='2021-09-01')]\n",
    "df_ood = df_ood.sort_values(by='DATE')\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "# df_ood = df_ood.loc[:2000]\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1.isna()].index)\n",
    "df_ood = df_ood.drop(df_ood.loc[df_ood.verbA0A1=='[]'].index)\n",
    "for col in ['verb','A0','A1']:\n",
    "  df_ood[col] = df_ood[col].apply(string_to_tuples_list)\n",
    "\n",
    "for col in ['stock_factors','verbA0A1']:\n",
    "  df_ood[col] = df_ood[col].apply(ast.literal_eval)\n",
    "df_ood = mask(df_ood)\n",
    "df_ood = df_ood.reset_index(drop=True)\n",
    "ood_data_loader = create_data_loader(df_ood, tokenizer, max_len, BATCH_SIZE)\n",
    "\n",
    "y_ood_review_texts, y_ood_pred, y_ood_pred_probs, y_ood = get_predictions(\n",
    "  model,\n",
    "  ood_data_loader\n",
    ")\n",
    "\n",
    "print(classification_report(y_ood, y_ood_pred, target_names=class_names,digits=4))\n",
    "\n",
    "ood_cm = confusion_matrix(y_ood, y_ood_pred)\n",
    "df_ood_cm = pd.DataFrame(ood_cm, index=class_names, columns=class_names)\n",
    "show_confusion_matrix(df_ood_cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame([y_ood_review_texts, y_ood_pred.numpy(), y_ood_pred_probs.numpy(), y_ood.numpy()]).T\n",
    "df = df.rename(columns={0:'text',1:'prediction',2:'probability',3:'labels'})\n",
    "df.to_csv('Factor24_Pretrained_RoBert_SRL(FC)_A0A1.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 22,
     "status": "aborted",
     "timestamp": 1638340935433,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "6QZ0EgQoalz2",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# !pip install lit-nlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 23,
     "status": "aborted",
     "timestamp": 1638340935434,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "eAeHdVF7c-QF",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# from lit_nlp.api.dataset import Dataset\n",
    "# class MultiNLIData(Dataset):\n",
    "#   \"\"\"Loader for MultiNLI development set.\"\"\"\n",
    "#   def __init__(self, df):\n",
    "#     # Read the eval set from a .tsv file as distributed with the GLUE benchmark.\n",
    "#     # df = pandas.read_csv(path, sep='\\t')\n",
    "#     # Store as a list of dicts, conforming to self.spec()\n",
    "#     self.LABELS = [0,1]\n",
    "#     self._examples = [{\n",
    "#       'sentence': row['DESCRIPTION_EN'],\n",
    "#       # 'hypothesis': row['sentence2'],\n",
    "#       'label': row['LABEL']\n",
    "#       # 'genre': row['genre'],\n",
    "#     } for _, row in df.iterrows()]\n",
    "#   def spec(self):\n",
    "#     return {\n",
    "#       'sentence': lit_types.TextSegment(),\n",
    "#       # 'hypothesis': lit_types.TextSegment(),\n",
    "#       'label': lit_types.CategoryLabel(vocab=self.LABELS),\n",
    "#       # We can include additional fields, which don't have to be used by the model.\n",
    "#       # 'genre': lit_types.Label(),\n",
    "#     }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 23,
     "status": "aborted",
     "timestamp": 1638340935434,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "l5886G4mdXia",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 24,
     "status": "aborted",
     "timestamp": 1638340935435,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "mrSBCdqO0sUs",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 24,
     "status": "aborted",
     "timestamp": 1638340935435,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "f-nX4zrOdxpo",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 25,
     "status": "aborted",
     "timestamp": 1638340935436,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "ARdnz5td18VV",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 25,
     "status": "aborted",
     "timestamp": 1638340935436,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "iMviJZsfdifG",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 25,
     "status": "aborted",
     "timestamp": 1638340935436,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "8h5I2QKodj8b",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 26,
     "status": "aborted",
     "timestamp": 1638340935437,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "0-abzg6avOQq",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 26,
     "status": "aborted",
     "timestamp": 1638340935437,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "flJ1xtWNvYej",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 25,
     "status": "aborted",
     "timestamp": 1638340935437,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "nBxb2zhWvfaK",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# from absl import app\n",
    "# from absl import flags\n",
    "# from absl import logging\n",
    "\n",
    "# from lit_nlp import dev_server\n",
    "# from lit_nlp import server_flags\n",
    "# from lit_nlp.api import model as lit_model\n",
    "# from lit_nlp.api import types as lit_types\n",
    "# # Use the regular GLUE data loaders, because these are very simple already.\n",
    "# from lit_nlp.examples.datasets import glue\n",
    "# from lit_nlp.lib import utils\n",
    "\n",
    "# import torch\n",
    "# import transformers\n",
    "\n",
    "# # NOTE: additional flags defined in server_flags.py\n",
    "\n",
    "# FLAGS = flags.FLAGS\n",
    "\n",
    "# flags.DEFINE_string(\n",
    "#     \"model_path\",\n",
    "#     \"https://storage.googleapis.com/what-if-tool-resources/lit-models/sst2_tiny.tar.gz\",\n",
    "#     \"Path to trained model, in standard transformers format, e.g. as \"\n",
    "#     \"saved by model.save_pretrained() and tokenizer.save_pretrained()\")\n",
    "\n",
    "\n",
    "# def _from_pretrained(cls, *args, **kw):\n",
    "#   \"\"\"Load a transformers model in PyTorch, with fallback to TF2/Keras weights.\"\"\"\n",
    "#   try:\n",
    "#     return cls.from_pretrained(*args, **kw)\n",
    "#   except OSError as e:\n",
    "#     logging.warning(\"Caught OSError loading model: %s\", e)\n",
    "#     logging.warning(\n",
    "#         \"Re-trying to convert from TensorFlow checkpoint (from_tf=True)\")\n",
    "#     return cls.from_pretrained(*args, from_tf=True, **kw)\n",
    "\n",
    "\n",
    "# class SimpleSentimentModel(lit_model.Model):\n",
    "#   \"\"\"Simple sentiment analysis model.\"\"\"\n",
    "\n",
    "#   LABELS = [0,1]  # negative, positive\n",
    "\n",
    "#   def __init__(self, model_name_or_path):\n",
    "#     self.tokenizer = transformers.AutoTokenizer.from_pretrained(\n",
    "#         model_name_or_path)\n",
    "#     model_config = transformers.AutoConfig.from_pretrained(\n",
    "#         model_name_or_path,\n",
    "#         num_labels=2,\n",
    "#         output_hidden_states=True,\n",
    "#         output_attentions=True,\n",
    "#     )\n",
    "#     # This is a just a regular PyTorch model.\n",
    "#     self.model = _from_pretrained(\n",
    "#         transformers.AutoModelForSequenceClassification,\n",
    "#         model_name_or_path,\n",
    "#         config=model_config)\n",
    "#     self.model.eval()\n",
    "\n",
    "#   ##\n",
    "#   # LIT API implementation\n",
    "#   def max_minibatch_size(self):\n",
    "#     # This tells lit_model.Model.predict() how to batch inputs to\n",
    "#     # predict_minibatch().\n",
    "#     # Alternately, you can just override predict() and handle batching yourself.\n",
    "#     return 16\n",
    "\n",
    "#   def predict_minibatch(self, inputs):\n",
    "#     # Preprocess to ids and masks, and make the input batch.\n",
    "#     encoded_input = self.tokenizer.batch_encode_plus(\n",
    "#         [ex[\"sentence\"] for ex in inputs],\n",
    "#         return_tensors=\"pt\",\n",
    "#         add_special_tokens=True,\n",
    "#         max_length=200,\n",
    "#         padding=\"longest\",\n",
    "#         truncation=\"longest_first\")\n",
    "\n",
    "#     # Check and send to cuda (GPU) if available\n",
    "#     if torch.cuda.is_available():\n",
    "#       self.model.cuda()\n",
    "#       for tensor in encoded_input:\n",
    "#         encoded_input[tensor] = encoded_input[tensor].cuda()\n",
    "#     # Run a forward pass.\n",
    "#     with torch.no_grad():  # remove this if you need gradients.\n",
    "#       out: transformers.modeling_outputs.SequenceClassifierOutput = \\\n",
    "#           self.model(**encoded_input)\n",
    "\n",
    "#     # Post-process outputs.\n",
    "#     batched_outputs = {\n",
    "#         \"probas\": torch.nn.functional.softmax(out.logits, dim=-1),\n",
    "#         \"input_ids\": encoded_input[\"input_ids\"],\n",
    "#         \"ntok\": torch.sum(encoded_input[\"attention_mask\"], dim=1),\n",
    "#         \"cls_emb\": out.hidden_states[-1][:, 0],  # last layer, first token\n",
    "#     }\n",
    "#     # Return as NumPy for further processing.\n",
    "#     detached_outputs = {k: v.cpu().numpy() for k, v in batched_outputs.items()}\n",
    "#     # Unbatch outputs so we get one record per input example.\n",
    "#     for output in utils.unbatch_preds(detached_outputs):\n",
    "#       ntok = output.pop(\"ntok\")\n",
    "#       output[\"tokens\"] = self.tokenizer.convert_ids_to_tokens(\n",
    "#           output.pop(\"input_ids\")[1:ntok - 1])\n",
    "#       yield output\n",
    "\n",
    "#   def input_spec(self) -> lit_types.Spec:\n",
    "#     return {\n",
    "#         \"sentence\": lit_types.TextSegment(),\n",
    "#         \"label\": lit_types.CategoryLabel(vocab=self.LABELS, required=False)\n",
    "#     }\n",
    "\n",
    "#   def output_spec(self) -> lit_types.Spec:\n",
    "#     return {\n",
    "#         \"tokens\": lit_types.Tokens(),\n",
    "#         \"probas\": lit_types.MulticlassPreds(parent=\"label\", vocab=self.LABELS),\n",
    "#         \"cls_emb\": lit_types.Embeddings()\n",
    "#     }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 26,
     "status": "aborted",
     "timestamp": 1638340935438,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "B64IFx5g461l",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# dataset = MultiNLIData(df_train.sample(1800))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 26,
     "status": "aborted",
     "timestamp": 1638340935438,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "qR0nbr94v2xz",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# datasets = {\"sst_dev\": dataset}\n",
    "# models = {\"sst\": SimpleSentimentModel(\"/content/drive/MyDrive/model/best_model/\")}\n",
    "# # models = {\"sst\": SimpleSentimentModel(\"bert-base-uncased\")}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 26,
     "status": "aborted",
     "timestamp": 1638340935438,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "WR8VwsH9v4l8",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# from lit_nlp import notebook\n",
    "# widget = notebook.LitWidget(models, datasets, height=800)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 27,
     "status": "aborted",
     "timestamp": 1638340935439,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "wNPctrXDBQSM",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# mm = models[\"sst\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 27,
     "status": "aborted",
     "timestamp": 1638340935439,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "lHX6hJtVCAuX",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# mm.model.config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 27,
     "status": "aborted",
     "timestamp": 1638340935439,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "cihtCCxC3ftN",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# widget.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 28,
     "status": "aborted",
     "timestamp": 1638340935440,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "h5c4S99U5UOS",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "# dataset.LABELS\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 28,
     "status": "aborted",
     "timestamp": 1638340935440,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "cKU3zawTXeiw",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "df =pd.read_csv('../data/df_ood_srl.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 29,
     "status": "aborted",
     "timestamp": 1638340935441,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "5uQ49ejLXi1s",
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 29,
     "status": "aborted",
     "timestamp": 1638340935441,
     "user": {
      "displayName": "Nero Cao",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s64",
      "userId": "05305083952686219705"
     },
     "user_tz": -630
    },
    "id": "97GdOVjpXonm",
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Factor24_Pretrained_RoBert_SRL(FC)_V_(MLP+QK).ipynb",
   "provenance": [
    {
     "file_id": "18JleubaGQc9sqy2lzjUadPBYpoXenzcx",
     "timestamp": 1638322241422
    },
    {
     "file_id": "1FG4HDOAZRGjMWtRAEMSNEhXUKozfQkop",
     "timestamp": 1638322191022
    },
    {
     "file_id": "1Vt9szQdPWB6WqQf5L9xSZs-MuMuNy47U",
     "timestamp": 1636702060132
    },
    {
     "file_id": "1hH9AIVD69cKbJFUaj47yk0pWyCmMdV3Y",
     "timestamp": 1635156978287
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "040c33afcdff47fcb1b807543c48eeb1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0aa1acd28f504292b91d7c32ee7d64ab": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0fdde7e1bc0f49c7a2a54364588408fb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "102f2b2c193340c4bbb10595aae790c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_3fd4f746e7cf4bdcb5b85322ff8eb5b8",
      "placeholder": "​",
      "style": "IPY_MODEL_da2eb0187a44429696ec150e27a7c559",
      "value": " 112/112 [00:00&lt;00:00, 3.08kB/s]"
     }
    },
    "1205434242214938b9b58ccf2f940300": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a74e5e1c50254c5ab459718dc2c59fb7",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_7f27bc2760854988ae083dcdd738c33c",
      "value": 2
     }
    },
    "1636be77b6e447af9fe0795d63620d44": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_36a3d6d220f8419ba1482110bac0cbe4",
      "placeholder": "​",
      "style": "IPY_MODEL_9a433ec7dd0a4626b1f28e41804f0e5f",
      "value": " 19.0/19.0 [00:00&lt;00:00, 607B/s]"
     }
    },
    "182deebee91d429db286768ffd8f330b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c66a9e829f74978a6dc6c5831c26d66": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ad27528a6d144adfbe3dbc276ee43298",
       "IPY_MODEL_1205434242214938b9b58ccf2f940300",
       "IPY_MODEL_af1d28bcf3814a978da6cb77bfe2537c"
      ],
      "layout": "IPY_MODEL_040c33afcdff47fcb1b807543c48eeb1"
     }
    },
    "233a52b14ada464a993ec8aee9d31f29": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "25d2a9565b5f422f980da2f0dde46232": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2758357c443e4c6aa54b11cd7aed859b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2c95ebb1f47a4113b9eba6753e248081": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3671d85e40434594b04539775f8f52dd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0fdde7e1bc0f49c7a2a54364588408fb",
      "placeholder": "​",
      "style": "IPY_MODEL_5575420737784e09916ad4194a761dac",
      "value": " 269k/269k [00:00&lt;00:00, 663kB/s]"
     }
    },
    "36a3d6d220f8419ba1482110bac0cbe4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "37984a156b634c668ac59e1ee50ab3c4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3aa458d8906e48daace7aed0292bd6f3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3fd4f746e7cf4bdcb5b85322ff8eb5b8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "44767259a6da4a82bdbdae29787c52c9": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eaffde3cf6ff4ee0a8e1aeade4a2db8e",
      "placeholder": "​",
      "style": "IPY_MODEL_83082bfcc08e43578a245146fd226427",
      "value": "Downloading: 100%"
     }
    },
    "46695f3c7b3b443194eac3d83918c812": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_66a007fb557848aab6f41102a1b8305f",
      "max": 19,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ab071e2b77a84e41a17b95671a3b1ad8",
      "value": 19
     }
    },
    "4f903879853d4bdb89a83608e71b79b5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5575420737784e09916ad4194a761dac": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "590af74a6be4442e8daa38c85df146e0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "59bbb69619154d6f807d924cbd057972": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "66a007fb557848aab6f41102a1b8305f": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "676a15c6b28e47f69d65e837314ac142": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "69e640e5d0eb447c8f4eb78cfe538e32": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "6abc245ff8454df4a8f82ba73b8b9dc8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_b9422f74176244e69952d394491fec8e",
       "IPY_MODEL_fe53e4c52b8f4ab198d221f2432b8798",
       "IPY_MODEL_102f2b2c193340c4bbb10595aae790c5"
      ],
      "layout": "IPY_MODEL_ba73aa97f348474185b341310767fb21"
     }
    },
    "6f4e06b72b5b445a8e7402e8b269b063": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_914e53be08704e638acbdf37f5d06dcc",
      "placeholder": "​",
      "style": "IPY_MODEL_a0b7ea994efa410bb22796fedf6af172",
      "value": "Downloading: 100%"
     }
    },
    "7c07929736284b658849f60af1b674d3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_25d2a9565b5f422f980da2f0dde46232",
      "max": 268961,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_69e640e5d0eb447c8f4eb78cfe538e32",
      "value": 268961
     }
    },
    "7f27bc2760854988ae083dcdd738c33c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "83082bfcc08e43578a245146fd226427": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "914e53be08704e638acbdf37f5d06dcc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9a433ec7dd0a4626b1f28e41804f0e5f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9bdfbf15bf7345f5847736122acacd7b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_2c95ebb1f47a4113b9eba6753e248081",
      "placeholder": "​",
      "style": "IPY_MODEL_676a15c6b28e47f69d65e837314ac142",
      "value": " 110k/110k [00:00&lt;00:00, 529kB/s]"
     }
    },
    "a0b7ea994efa410bb22796fedf6af172": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a2ac63819f2144c5b66b57132f1ff185": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ae07d83052cf46ae9c4231eebcb8c5d4",
      "max": 109540,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_b42983f2583e40c1bf3c398cef8e2cf5",
      "value": 109540
     }
    },
    "a74e5e1c50254c5ab459718dc2c59fb7": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "aa469eccce834677bff79b2d7080ea0f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ab071e2b77a84e41a17b95671a3b1ad8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ad27528a6d144adfbe3dbc276ee43298": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c1b1e62f31564c3d801a835ef6f723d1",
      "placeholder": "​",
      "style": "IPY_MODEL_3aa458d8906e48daace7aed0292bd6f3",
      "value": "Downloading: 100%"
     }
    },
    "ae07d83052cf46ae9c4231eebcb8c5d4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "af1d28bcf3814a978da6cb77bfe2537c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_590af74a6be4442e8daa38c85df146e0",
      "placeholder": "​",
      "style": "IPY_MODEL_c485b6952a4a421f89383b43b7002fd8",
      "value": " 2.00/2.00 [00:00&lt;00:00, 59.1B/s]"
     }
    },
    "b25c54179da848acaceec7f396da6b09": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_44767259a6da4a82bdbdae29787c52c9",
       "IPY_MODEL_7c07929736284b658849f60af1b674d3",
       "IPY_MODEL_3671d85e40434594b04539775f8f52dd"
      ],
      "layout": "IPY_MODEL_2758357c443e4c6aa54b11cd7aed859b"
     }
    },
    "b42983f2583e40c1bf3c398cef8e2cf5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "b8b6f228c2dc42519de4137d19f24e3a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b9422f74176244e69952d394491fec8e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_b8b6f228c2dc42519de4137d19f24e3a",
      "placeholder": "​",
      "style": "IPY_MODEL_4f903879853d4bdb89a83608e71b79b5",
      "value": "Downloading: 100%"
     }
    },
    "ba73aa97f348474185b341310767fb21": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c1b1e62f31564c3d801a835ef6f723d1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "c485b6952a4a421f89383b43b7002fd8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c4db4ab5d5b142c787da7e737f425050": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_6f4e06b72b5b445a8e7402e8b269b063",
       "IPY_MODEL_46695f3c7b3b443194eac3d83918c812",
       "IPY_MODEL_1636be77b6e447af9fe0795d63620d44"
      ],
      "layout": "IPY_MODEL_182deebee91d429db286768ffd8f330b"
     }
    },
    "cce639937ece4bd082588d5b592a4a0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f660e8b44e204795afd2d0449886e37a",
       "IPY_MODEL_a2ac63819f2144c5b66b57132f1ff185",
       "IPY_MODEL_9bdfbf15bf7345f5847736122acacd7b"
      ],
      "layout": "IPY_MODEL_0aa1acd28f504292b91d7c32ee7d64ab"
     }
    },
    "da2eb0187a44429696ec150e27a7c559": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "eaffde3cf6ff4ee0a8e1aeade4a2db8e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "f660e8b44e204795afd2d0449886e37a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_233a52b14ada464a993ec8aee9d31f29",
      "placeholder": "​",
      "style": "IPY_MODEL_37984a156b634c668ac59e1ee50ab3c4",
      "value": "Downloading: 100%"
     }
    },
    "fe53e4c52b8f4ab198d221f2432b8798": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_59bbb69619154d6f807d924cbd057972",
      "max": 112,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_aa469eccce834677bff79b2d7080ea0f",
      "value": 112
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
